'''
线上环境配置文件
'''
import os
import sys

BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, BASE_DIR)

BOT_NAME = 'amac_spider'

SPIDER_MODULES = ['amac_spider.spiders']
NEWSPIDER_MODULE = 'amac_spider.spiders'

# 使用scrapy_redis调度器
SCHEDULER = "scrapy_redis.scheduler.Scheduler"

# 使用scrapy_redis去重规则，实际上使用的还是scrapy的去重
DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"

# 使用scrapy_redis提供的先入先出队列
SCHEDULER_QUEUE_CLASS = 'scrapy_redis.queue.FifoQueue'

# 爬虫停止后，不清除去重队列
SCHEDULER_PERSIST = True

# Mysql配置信息(线上环境)
MYSQL_HOST = '127.0.0.1'
MYSQL_PORT = 3306
MYSQL_USER = 'root'
MYSQL_PASSWORD = 'root'
MYSQL_DB = 'amac_db'
MYSQL_CHARSET = 'utf8'

# Redis配置信息(线上环境)
REDIS_HOST = '127.0.0.1'
REDIS_PORT = 6379
REDIS_DB = 0
REDIS_USER = None
REDIS_PASSWORD = None

# 代理池API
PROXY_POOL_INTERFACE = 'http://127.0.0.1:5000/api/v1/proxy'

# Scrapy LOG日志级别
LOG_LEVEL = 'INFO'

# 默认重试次数
RETRY_TIMES = 5

# 是否遵循robots协议
ROBOTSTXT_OBEY = False

# 每次并发量，默认16个
# Configure maximum concurrent requests performed by Scrapy (default: 16)
# CONCURRENT_REQUESTS = 16

# 下载超时时间
# DOWNLOAD_TIMEOUT = 180

# 开启下载延迟扩展
# AUTOTHROTTLE_ENABLED = True

# 设置下载延迟(注：必须开启下载延迟扩展才可用)
# DOWNLOAD_DELAY = 1

# Redis链接URL
REDIS_URL = 'redis://{user}:{password}@{host}:{port}/{db}'.format(user=(REDIS_USER or ''),
                                                                  password=(REDIS_PASSWORD or ''),
                                                                  host=REDIS_HOST,
                                                                  port=REDIS_PORT,
                                                                  db=REDIS_DB)

DOWNLOADER_MIDDLEWARES = {
   'scrapy.downloadermiddlewares.retry.RetryMiddleware': None,
   'amac_spider.utils.my_middlewares.AddHeadersMiddleware': 1000,
   'amac_spider.utils.my_middlewares.GetProxyMiddleware': 1500,
   'amac_spider.utils.my_middlewares.MyRetryMiddleware': 10000,
}

ITEM_PIPELINES = {
   'amac_spider.pipelines.AmacspiderInsertMysqlPipeline': 300,
   # 'amac_spider.pipelines.TestPipeline': 300,
}

FUND_MANAGER_UNIQUE_KEY = {
    'list': 'manager_id',
    'detail': 'manager_id',
    'member_info': 'manager_id',
    'management_info': ['manager_id', 'name'],
    'legal_opinion_status': 'manager_id',
    'management_work_experience': ['manager_id', 'name', 'time', 'company_name', 'department'],
    'related_party_info': ['manager_id', 'record_code'],
    'promoter_info': ['manager_id', 'name',],
    'implement_before_found_fund': ['manager_id', 'company_name'],
    'implement_after_found_fund': ['manager_id', 'company_name'],
    'invest_adviser_product': ['manager_id', 'company_name'],
    'org_honesty_info': 'manager_id'
}

# 不同爬虫名称对应不同插入类和URL的映射
FIRST_URL_MAPPING = {
    'aoin_product':
        {
            'cls': 'AoinProductInsert',
            'url': 'https://gs.amac.org.cn/amac-infodisc/res/aoin/product',
            'unique_key': 'product_id'
         },
    'pof_subfund':
        {
            'cls': 'PofSubfundInsert',
            'url': 'https://gs.amac.org.cn/amac-infodisc/res/pof/subfund',
            'unique_key': 'product_id'
        },
    'cancelled_manager':
        {
            'cls': 'CancelledManagerInsert',
            'url': 'https://gs.amac.org.cn/amac-infodisc/res/cancelled/manager',
            'unique_key': 'code'
        },
    'pof_person':
        {
            'cls': 'PofPersonInsert',
            'url': 'https://gs.amac.org.cn/amac-infodisc/res/pof/person',
            'unique_key': {'list': ['org_id', 'org_type'], 'detail': 'person_id', 'cert_status_change_record': ['person_id', 'cert_code', 'cert_change_time']}
        },
    'pof_securities':
        {
            'cls': 'PofSecuritiesInsert',
            'url': 'https://gs.amac.org.cn/amac-infodisc/res/pof/securities',
            'unique_key': 'product_id'
        },
    'fund_account':
        {
            'cls': 'FundAccountInsert',
            'url': 'https://gs.amac.org.cn/amac-infodisc/res/fund/account',
            'unique_key': 'product_id'
        },
    'mutual_fund_manager':
        {
            'cls': 'MutualFundManagerInsert',
            'url': 'https://www.amac.org.cn/portal/front/mutualFund/findMutualFundHousePage',
            'unique_key': 'company_name'
        },
    'fund_manager':
        {
            'cls': 'FundManagerInsert',
            'url': 'https://gs.amac.org.cn/amac-infodisc/res/pof/manager',
            'unique_key': FUND_MANAGER_UNIQUE_KEY
        },
}

# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16


# Enable or disable spider middlewares
# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
#    'amac_spider.middlewares.AmacspiderSpiderMiddleware': 543,
#}


# Enable or disable extensions
# See https://docs.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
#    'scrapy.extensions.telnet.TelnetConsole': None,
#}


